#coding: utf8
import urlparse
import time
import os.path
import re
import sys
from pygeocoder import Geocoder
from decimal import *
from scrapy.http import Request
from scrapy.http import FormRequest
from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from fresh.items import RestaurantItem
from fresh.items import RestaurantReview
from fresh.TripMasterDB import TripMasterDB
import datetime

import MySQLdb
from fresh.utils import Utils
from fresh.TripUtils import TripUtils


class TripHistSpider(BaseSpider):

    handle_httpstatus_list = [400,404,410]
    handle_bad_request = [400]
    name = "triphist"
    allowed_domains = ["www.tripadvisor.es"]
    start_urls = []
    
    db = TripMasterDB()
    tripUtils = TripUtils()

    utils = Utils()
    
    logLocalName="TRIP_HIST_SCRAPPY"
    
    page=0
    numRestFound=0
    
    def __init__(self, *args, **kwargs):
        super(TripHistSpider, self).__init__()
        reload(sys)
        sys.setdefaultencoding('utf-8')
        time = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
        filename = time + '.log'
        filename = os.path.join('logTripMaster/',filename)
        self.utils.logProcessInit(filename)

        self.utils.logProcessBegin(self.logLocalName)
        
        self.db.utils=self.utils
        self.tripUtils.utils=self.utils
        if (self.db.connectDB()):
            self.db.readURLs(self.start_urls)


    def make_requests_from_url(self, url):
        idRestSource=None
        dtPost=None
        idRestSource,dtPost = self.db.getRestFromUrl(url)
        
        """return Request(url, dont_filter=True, meta = {'idRestSource': idRestSource})"""
        return FormRequest("http://www.tripadvisor.es/SetLangFilter",
                    formdata={'filterLang': 'ALL', 'returnTo': url,'sortOrder':'1'},
                    meta = {'idRestSource': idRestSource,'dtPost':dtPost,'first':'1'},
                    dont_filter=True,
                    callback=self.parse)
        
        """
        return FormRequest("http://www.tripadvisor.es/SortEateries",
                    formdata={'preferredSegment': '-1', 'returnTo': url,'sortOrder':'1'},
                    meta = {'idRestSource': idRestSource},
                    dont_filter=True,
                    callback=self.parse)
        """

    
    def parse(self, response):
        try:

            #from scrapy.shell import inspect_response
            #inspect_response(response)
            
            restaurant=None
            hxs = HtmlXPathSelector(response)
            idRestSource = response.meta['idRestSource']
            dtPost = response.meta['dtPost']
            #url=response.meta['returnTo']          
            
            if idRestSource is None:
                self.utils.logProcessDetail("ERROR DE PROGRAMACIÓN/BD NO SE PUEDE OBTENER EL CODIGO DE RESTAURANTE A PARTIR DE URL: "+str(response.url) , self.utils.ERROR)
                return

            """check if the response is a not found"""
            if response.status in self.handle_httpstatus_list:               
                self.db.setRestaurantStatus(idRestSource,self.db.NACTIVE)
                return


            restaurant = RestaurantItem()
            restaurant['idRestSource'] = idRestSource
            restaurant['url_scrapy'] = response.url

            self.numRestFound=self.numRestFound+1

            self.tripUtils.fillRestaurant(response,hxs,restaurant)

            yield Request(response.url,callback=self.parse_1PagePosts,meta={'restaurant': restaurant,'dtPost':dtPost})


            
        except Exception,e:
            self.utils.logProcessDetail("Error: " + str(e)+ " parsing restaurant en TRIP: " + str(restaurant) ,self.utils.ERROR)


    def parse_1PagePosts(self,response):
        try:



            
            hxs = HtmlXPathSelector(response)
            restaurant = response.meta['restaurant']
            dtPost = response.meta['dtPost']
            #ids = hxs.select('//div/@class[contains(.,"reviewSelector")]/../../div/@id').extract()
            ids = hxs.select('//div/@class[contains(.,"reviewSelector")]/../@id').extract()
            if ids is not None and len(ids)>0:
                sIds = self.__getPostsIds(ids)
                self.utils.logProcessDetail("id: " + str(sIds),self.utils.DEBUG)
                url = 'http://www.tripadvisor.es/UserReviewController?a=rblock&r='+sIds+'&type=1&tr=false&n=16'
                #url='http://www.tripadvisor.es/UserReviewController?a=rblock&r=191537421&type=1&tr=false&n=16'

                #get next page in case needed to go further
                nextPageurl = hxs.select('//a[@class="guiArw sprite-pageNext "]/@href').extract()
                if nextPageurl is not None and len(nextPageurl)>0:
                    nextPageurl = urlparse.urljoin(response.url,nextPageurl[0])
                    #self.utils.logProcessDetail("url------------:"+str(nextPageurl),self.utils.INFO)
                yield Request(url,callback=self.parse_posts,meta={'restaurant': restaurant,'dtPost':dtPost,'nextPageurl':nextPageurl})

        except Exception,e:
            self.utils.logProcessDetail("Error: " + str(e) + " parse_1PagePosts en TRIP: " + str(restaurant) ,self.utils.ERROR)
    def __getPostsIds(self,ids):
        sIds=None
        bFirst=True
        for i in ids:
            if bFirst:
                sIds = str(i).replace("review_","")
            else:
                sIds = sIds + ":" + str(i).replace("review_","")            
            bFirst=False
        sIds=sIds.replace("fresh_reviews:","")
        return sIds

    def parse_posts(self, response):
        try:



            
            if response.status in self.handle_bad_request:
                self.utils.logProcessDetail("Error: bad request al parsear post",self.utils.ERROR)
                
            hxs = HtmlXPathSelector(response)
            restaurant = response.meta['restaurant']
            dtPost = response.meta['dtPost']
            nextPageurl = response.meta['nextPageurl']
            
            if dtPost is None:
                dLimitLow = datetime.datetime.strptime('01/01/2012', '%d/%m/%Y').date()
                #self.utils.logProcessDetail("DTPOST1: " + str(dtPost),self.utils.INFO)
            else:
                dLimitLow = dtPost.date()-datetime.timedelta(days=30)
                #self.utils.logProcessDetail("DTPOST2: " + str(dtPost),self.utils.INFO)

            restaurantReview=None
            for l in hxs.select('//div/@class[contains(.,"extended provider")]/..'):
                restaurantReview = self.tripUtils.fillRestaurantComment(response,l,restaurant)
            #self.utils.logProcessDetail("Restaurant: "+ str(restaurant),self.utils.DEBUG)
            #next page in case db last post is lower than last post scrapped
            if restaurantReview is not None:
                dDate = datetime.datetime.strptime(restaurantReview['date'], '%d/%m/%Y').date()
                #self.utils.logProcessDetail("0Limite inferior fecha DB: " +str(dLimitLow)+ " post scrapeado: " + str(dDate),self.utils.INFO)
                if dDate>dLimitLow and nextPageurl is not None and len(nextPageurl)>0:
                    yield Request(nextPageurl,callback=self.parse_1PagePosts,meta={'restaurant': restaurant,'dtPost':dtPost})
            yield restaurant
            
        except Exception,e:
            self.utils.logProcessDetail("Error: " + str(e) + " parse_restaurant en TRIP: " + str(restaurant) ,self.utils.ERROR)


    def parse_translation_post_custom(self, response):
        try:

            hxs = HtmlXPathSelector(response)
            restaurant = response.meta['restaurant']
            restaurantReview = response.meta['restaurantReview']
            
            self.tripUtils.fillRestaurantCommentTranslationCustom(response,hxs,restaurant,restaurantReview)

            return restaurant
        
        except Exception,e:
            self.utils.logProcessDetail("Error: " + str(e) + " parse_translation_post_custom en TRIP: " + str(restaurant),self.utils.ERROR)

    def parse_translation_post_google(self, response):
        try:

            hxs = HtmlXPathSelector(response)
            restaurant = response.meta['restaurant']
            restaurantReview = response.meta['restaurantReview']
            
            self.tripUtils.fillRestaurantCommentTranslationGoogle(response,hxs,restaurant,restaurantReview)

            return restaurant

        except Exception,e:
            self.utils.logProcessDetail("Error: " + str(e) + " parse_translation_post_google en TRIP: " + str(restaurant) ,self.utils.ERROR)


    def __getTripTranslationURL(self, url):
        try:
            po = url.find("/MachineTranslation")
            url = url[po:]
            
            po = url.find("sl=")
            tmpsl = url[po:]
            po = tmpsl.find("&")
            sl = tmpsl[:po]
            #self.utils.logProcessDetail("source language: " + str(sl) ,self.utils.INFO)

            po = url.find("tl=")
            tl = url[po:]
            #self.utils.logProcessDetail("source language: " + str(sl) ,self.utils.INFO)

            url = url.replace("tl="+tl,"tl="+sl)

            return url

        except Exception,e:
            self.utils.logProcessDetail("Error __getTripTranslationURL en TRIP: " + str(e) ,self.utils.ERROR)



    def close_spider(self):
        self.utils.logProcessEnd("Robot TRIP found: "+str(self.numRestFound) , self.utils.DEBUG)

            
        